package action;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;

import main.RequestManager;

import org.cyberneko.html.parsers.DOMParser;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.Node;
import org.dom4j.io.DOMReader;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import utils.Logger;
import utils.ThreadPoolRunnable;

import com.gargoylesoftware.htmlunit.ScriptException;

public class WebSourceCollector implements ThreadPoolRunnable {
    public static final String ALL_CHAR = "ABCDEFGHIJKLMNOPQRSTUVWXYZÀÁÂÃÈÉÊÌÍÒÓÔÕÙÚÝàáâãèéêìíòóôõùúýĂăĐđĨĩŨũƠơƯưẠạẢảẤấẦầẨẩẪẫẬậẮắẰằẲẳẴẵẶặẸẹẺẻẼẽẾếỀềỂểỄễỆệỈỉỊịỌọỎỏỐốỒồỔổỖỗỘộỚớỜờỞởỠỡỢợỤụỦủỨứỪừỬửỮữỰự";
    public static final String REPLACE_CHAR = "abcdefghijklmnopqrstuvwxyzaaaaeeeiioooouuyaaaaeeeiioooouuuaaddiiuuoouuaaaaaaaaaaaaaaaaaaaaaaaaeeeeeeeeeeeeeeeeiiiioooooooooooooooooooooooouuuuuuuuuuuuuu";
    
    public static String[] URL_IGNORE_LIST = {"google.com", "googleusercontent.com", "gstatic.com", "schema.org", ".js", ".css", ".dtd"};
    public static String[] URL_BREAK_CHAR = new String[]{"\"", "\'", "<", " ", "\n", ")", "("};
    public static List<String> NODE_NAME_IGNORE_LIST = Arrays.asList("SCRIPT", "STYLE");
    public static List<String> NODE_NAME_GET_PARENT_CONTENT_LIST = Arrays.asList("SPAN", "STRONG", "FONT", "B", "I", "U", "EM", "PRE");
    
    public static List<String> ERROR_ENDCODE_CHARACTER_LIST = Arrays.asList("»", "�");
    
    public static HashMap<String, String> knownUrls = new HashMap<String, String>();
    public static HashMap<String, String> knownResultTexts = new HashMap<String, String>();
    
    public static int totalPage = 0;
    public static int processPageCount = 0;
    
    public String url;
    public String keyword;
    private StringBuffer textBuffer = new StringBuffer();
    private StringBuffer imageBuffer = new StringBuffer();
    private StringBuffer linkBuffer = new StringBuffer();
    public Logger log;
    public Logger imageLog;
    public Logger linkLog;
    public boolean isGoogleResultLink = true;
    public boolean isStreamMode = true;
    public int keyFoundCount;
    public int imageFoundCount;
    public int linkFoundCount;
    public long startTime = System.currentTimeMillis();
    public int gatherDepth;
    public int currentDepth;
    
    private WebDriver driver;
    private BufferedReader in;
    
    public WebSourceCollector(String url, String keyword, Logger log, Logger imageLog, Logger linkLog, int gatherDepth, int currentDepth) {
        this.url = url;
        this.keyword = keyword;
        this.log = log;
        this.imageLog = imageLog;
        this.linkLog = linkLog;
        this.gatherDepth = gatherDepth;
        this.currentDepth = currentDepth;
    }
    
    @Override
    public Object[] getInitData() {
        return null;
    }

    private String getUrlFromGoogleSearchResultLink(String googleLink) {
        String data = requestDataByStream(googleLink);
        List<String> subUrls = getUrlFromGoogleSource(data);
        if (subUrls.size() > 0) {
            String googleRedirectUrl = subUrls.get(0);
            if (googleRedirectUrl.lastIndexOf("\"") > - 1) {
                googleRedirectUrl = googleRedirectUrl.substring(0, googleRedirectUrl.lastIndexOf("\""));
            }
            
            data = requestDataByStream(googleRedirectUrl);
            subUrls = getUrlFromSource(data);
            if (subUrls.size() > 0) {
                return subUrls.get(0);
            }
        }
        return null;
    }

    @Override
    public void runIt(Object[] thData) {
        startTime = System.currentTimeMillis();
        if (isGoogleResultLink) {
            String subUrl = getUrlFromGoogleSearchResultLink(url);
            if (subUrl != null) {
                url = subUrl;
                isGoogleResultLink = false;
            } else {
                return;
            }
        }
        
        String data = "";
        totalPage++;
        textBuffer.append(url + "\n");
        imageBuffer.append(url + "\n<br>");
        linkBuffer.append(url + "\n<br>");
        
        if (!isStreamMode) {
            data = requestDataNoJs(url);
        } else {
            totalPage++;
            data = requestDataByStream(url);
        }
        
        findKeyword(data, keyword);
        
        boolean isFoundData = false;
        if (keyFoundCount > 0) {
            textBuffer.append("===================================");
            log.info(textBuffer.toString(), false);
            log.flush();
            isFoundData = true;
        }
        
        if (imageFoundCount > 0) {
            imageBuffer.append("===================================<br>");
            imageLog.info(imageBuffer.toString(), false);
            imageLog.flush();
            isFoundData = true;
        }
        
        if (linkFoundCount > 0) {
            linkBuffer.append("===================================<br>");
            linkLog.info(linkBuffer.toString(), false);
            linkLog.flush();
            isFoundData = true;
        }
        
        if (isFoundData) {
            System.out.println(String.format("Finished [%s/%s] [%s/%s] %s", currentDepth, gatherDepth, ++processPageCount, totalPage, url));
        } else {
            if (isGoogleResultLink) {
                restartCollector(this);
            }
            totalPage--;
        }
    }
    
    public boolean isKnownResult(String text) {
        if (text == null) {
            return false;
        }
        return knownResultTexts.get(text) != null;
    }
    
    @SuppressWarnings("unchecked")
    public void findKeyword(String source, String keyword) {
        DOMParser parser = new DOMParser();
        InputSource inputSource = new InputSource(new ByteArrayInputStream(source.getBytes()));
        try {
            parser.parse(inputSource);
            org.w3c.dom.Document doc = parser.getDocument();
            DOMReader reader = new DOMReader();
            Document document = reader.read(doc);
            
            // Print all relate content to the keyword
            List<Node> nodes = document.selectNodes("//text()[contains(translate(., '" + ALL_CHAR + "', '" + REPLACE_CHAR + "'),'" + keyword.toLowerCase() + "')]");
            for (Node node : nodes) {
                if (!NODE_NAME_IGNORE_LIST.contains(node.getParent().getQualifiedName())) {
                    String parent = node.getParent().getQualifiedName();
                    String text = trimAll(node.getText());
                    
                    if (NODE_NAME_GET_PARENT_CONTENT_LIST.contains(node.getParent().getQualifiedName())) {
                        parent = node.getParent().getParent().getQualifiedName();
                        text = trimAll(node.getParent().getText());
                    }
                    
                    if (text != null && !isKnownResult(text)) {
                        knownResultTexts.put(text, text);
                        if ("A".equals(node.getParent().getQualifiedName())) {
                            Element element = (Element) node.getParent();
                            String href = completeUrl(element.attributeValue("href"), url);
                            if (href != null && knownUrls.get(href) == null && !detectErrorEncode(text)) {
                                knownUrls.put(href, href);
                                linkBuffer.append(String.format("<a href=\"%s\">%s</a><br>\n", href, text));
                                linkFoundCount++;
                                if (currentDepth < gatherDepth) {
                                    WebSourceCollector sourceCollector = new WebSourceCollector(href, keyword, log, imageLog, linkLog, gatherDepth, currentDepth + 1);
                                    sourceCollector.isGoogleResultLink = false;
                                    RequestManager.instance().addRequest(sourceCollector);
                                }
                            }
                        } else {
                            if (!detectErrorEncode(text)) {
                                textBuffer.append("[" + parent + "] " +  text + "\n");
                                keyFoundCount++;
//                            } else {
//                                System.out.println("Detected error encode in page: " +  url);
                            }
                        }
                    }
                }
            }
            
            // Print all the image url
            nodes = document.selectNodes("//IMG");
            for (Node node : nodes) {
                Element element = (Element) node;
                String src = completeUrl(element.attributeValue("src"), url);
                if ((src != null) && isContentAllKeyword(src, keyword) && !isKnownResult(src)) {
                    knownResultTexts.put(src, src);
                    imageBuffer.append(String.format("<img src=\"%s\"/><br>\n", src));
                    imageFoundCount++;
                }
            }
        } catch (SAXException e) {
        } catch (IOException e) {
        }
    }
    
    private String completeUrl(String url, String currentUrl) {
        if (url == null) {
            return null;
        }
        
        // Get dowmain
        int index = currentUrl.lastIndexOf("/");
        if (index > -1) {
            currentUrl = currentUrl.substring(0, index);
        }
        
        if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("//")) {
            return url;
        }
        
        if (url.startsWith("/")) {
            return currentUrl + url;
        }
        return currentUrl + "/" + url;
    }
    
    private boolean isContentAllKeyword(String source, String keyword) {
        if (source == null || keyword == null) {
            return false;
        }
        
        String[] keywords = keyword.toLowerCase().split(" ");
        for (int i = 0; i < keywords.length; i++) {
            if (!source.contains(keywords[i].trim())) {
                return false;
            }
        }
        return true;
    }
    
    public void close() {
        if (driver != null) {
            driver.close();
            driver = null;
        }
        
        if (in != null) {
            try {
                in.close();
            } catch (IOException e) {
            }
            in = null;
        }
    }
    
    protected String requestDataExecuteJs(String url) {
        try {
            driver = new HtmlUnitDriver(true);
            try {
                driver.get(url);
            } catch (ScriptException ex) {
                driver.close();
                return requestDataByStream(url);
            }
            String source = driver.getPageSource();
            driver.close();
            driver = null;
            return source;
        } catch (Exception ex) {
            return "";
        }
    }
    
    protected String requestDataNoJs(String url) {
        driver = new HtmlUnitDriver();
        driver.get(url);
        String source = driver.getPageSource();
        driver.close();
        driver = null;
        return source;
    }
    
    protected String requestDataByStream(String url) {
        StringBuffer buff = new StringBuffer();
        try {
            URL urlObj = new URL(url);
            URLConnection conn = urlObj.openConnection();
            conn.setConnectTimeout(60000);
            conn.setReadTimeout(60000);
            in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
            String inputLine;
            while ((inputLine = in.readLine()) != null) {
                buff.append(inputLine);
                buff.append("\n");
            }
        } catch (Exception e) {
        } finally {
            try {
                if (in != null) {
                    in.close();
                    in = null;
                }
            } catch (IOException e) {
            }
        }
        return buff.toString();
    }
    
    private static boolean detectErrorEncode(String source) {
        for (String errorEncodeText : ERROR_ENDCODE_CHARACTER_LIST) {
            if (source.indexOf(errorEncodeText) > -1) {
                return true;
            }
        }
        return false;
    }
    
    private static int nextUrlBreakCharIndex(String source, int fromIndex) {
        int index = -1;
        for (int i = 0; i < URL_BREAK_CHAR.length; i++) {
            int idx = source.indexOf(URL_BREAK_CHAR[i], fromIndex);
            if (index < 0 || (idx < index && idx > 0)) {
                index = idx;
            }
        }
        return index;
    }
    
    private static String trimAll(String text) {
        if (text == null) {
            return null;
        }
        text = text.replaceAll("(\\r|\\n)", "");
        text = text.trim();
        return text;
    }
    
    public static List<String> getUrlFromGoogleSource(String source) {
        String urlBeginString = "/url?q=";
        int index = source.indexOf(urlBeginString);
        List<String> urls = new ArrayList<String>();
w:      while (index > 0) {
            int urlBreakIndex = nextUrlBreakCharIndex(source, index + 1);
            String url = "";
            if (urlBreakIndex < 0) {
                url = source.substring(index);
            } else {
                url = source.substring(index, urlBreakIndex);
            }
            
            for (String ignore : URL_IGNORE_LIST) {
                if (url.indexOf(ignore) > -1) {
                    index = source.indexOf(urlBeginString, urlBreakIndex + 1);
                    continue w;
                }
            }
            
            String fullUrl = "https://www.google.com.vn" + url;
            if (knownUrls.get(fullUrl) == null) {
                urls.add(fullUrl);
                knownUrls.put(fullUrl, fullUrl);
            }
           
            index = source.indexOf(urlBeginString, urlBreakIndex + 1);
        }
        return urls;
    }
    
    public static List<String> getUrlFromSource(String source) {
        String urlBeginString = "http://";
        int index = source.indexOf(urlBeginString);
        List<String> urls = new ArrayList<String>();
w:      while (index > 0) {
            int urlBreakIndex = nextUrlBreakCharIndex(source, index + 1);
            String url = "";
            if (urlBreakIndex < 0) {
                url = source.substring(index);
            } else {
                url = source.substring(index, urlBreakIndex);
            }
            
            for (String ignore : URL_IGNORE_LIST) {
                if (url.indexOf(ignore) > -1) {
                    index = source.indexOf(urlBeginString, urlBreakIndex + 1);
                    continue w;
                }
            }
            
            if (knownUrls.get(url) == null) {
                urls.add(url);
                knownUrls.put(url, url);
            }
            index = source.indexOf(urlBeginString, urlBreakIndex + 1);
        }
        return urls;
    }
    
    public static void processResultFromGoogle(String source, String keyword, Logger log, Logger imageLog, Logger linkLog, int gatherDepth) {
        List<String> urls = getUrlFromGoogleSource(source);
        for (String url : urls) {
            RequestManager.instance().addRequest(new WebSourceCollector(url, keyword, log, imageLog, linkLog, gatherDepth, 0));
        }
    }
    
    public static void restartCollector(WebSourceCollector sourceCollector) {
        WebSourceCollector newSourceCollector = new WebSourceCollector(sourceCollector.url, sourceCollector.keyword, sourceCollector.log, sourceCollector.imageLog, sourceCollector.linkLog, sourceCollector.gatherDepth, sourceCollector.currentDepth);
        newSourceCollector.isGoogleResultLink = sourceCollector.isGoogleResultLink;
        newSourceCollector.isStreamMode = false;
        RequestManager.instance().addRequest(newSourceCollector);
        System.out.println("Restart for url: " + sourceCollector.url);
    }
}
